package main

import (
	"../utils/proxyutil"
	"../utils/strutil"
	"fmt"
	"github.com/gocolly/colly"
	"github.com/gocolly/colly/extensions"
	"strings"
)

type JuZiBean struct {
	Category string `json:"category"`
	Title    string `json:"title"`
	Content  string `json:"content"`
}

func main() {

	bUrl := "http://www.qqzf.cn"

	proxyList := proxyutil.GetProxyList()

	c := colly.NewCollector()
	extensions.RandomUserAgent(c)
	extensions.Referer(c)

	/*分类*/
	c.OnHTML("li", func(e *colly.HTMLElement) {
		_requestUrl := fmt.Sprintf("%v", e.Request.URL)
		if bUrl == _requestUrl {
			e.ForEach(".item-color", func(i int, el *colly.HTMLElement) {
				_a := el.DOM
				_title := strutil.CoverGBKToUTF8(_a.Text())
				_link, _ := _a.Attr("href")
				_newLink := fmt.Sprintf("%s%s", bUrl, _link)
				fmt.Println(_title, _newLink)

				c.SetProxy(proxyutil.GetRandomProxy(proxyList))
				c.Visit(_newLink)
			})
		}
	})

	/*列表*/
	c.OnHTML(".left-item", func(e *colly.HTMLElement) {
		e.ForEach("h3", func(i int, el *colly.HTMLElement) {
			_a := el.DOM.Find("a")
			_title := strutil.CoverGBKToUTF8(_a.Text())
			_link, _ := _a.Attr("href")
			_newLink := fmt.Sprintf("%s%s", bUrl, _link)
			fmt.Println(_title, _newLink)

			c.SetProxy(proxyutil.GetRandomProxy(proxyList))
			c.Visit(_newLink)
		})
	})

	/*正文*/
	c.OnHTML(".article-left", func(e *colly.HTMLElement) {

		//正文分类
		_class := strutil.TrimHtml(strutil.CoverGBKToUTF8(e.DOM.Find(".list-title").Eq(0).Text()))
		_classSplit := strings.Split(_class, ">")
		if strings.HasSuffix(_class, ">") {
			_class = _classSplit[len(_classSplit)-2]
		} else {
			_class = _classSplit[len(_classSplit)-1]
		}

		//正文标题
		_title := strutil.TrimHtml(strutil.CoverGBKToUTF8(e.DOM.Find(".title").Eq(0).Text()))

		//fmt.Println(_class, _title)

		//正文内容
		e.ForEach(".ac-content>p", func(i int, el *colly.HTMLElement) {
			_a := el.DOM
			_content := strutil.TrimHtml(strutil.CoverGBKToUTF8(_a.Text()))
			if strings.Contains(_content, "、") {
				juziBean := JuZiBean{}
				juziBean.Category = _class
				juziBean.Title = _title
				juziBean.Content = strutil.TrimSerialNumber(_content)
				Save(juziBean)
			}
		})
	})

	c.OnRequest(func(r *colly.Request) {
		//fmt.Println("链接", r.URL)
	})

	c.SetProxy(proxyutil.GetRandomProxy(proxyList))

	//c.Visit("http://www.qqzf.cn/lizhijuzi/78880.html")
	//c.Visit("http://www.qqzf.cn/lizhijuzi/")
	c.Visit("http://www.qqzf.cn")

}

/**
保存数据
*/
func Save(bean JuZiBean) {
	fmt.Println(bean)
}
